{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "35af9c05-29dd-43cc-af3d-dc2421ed3961",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "08d0f459-e44b-4ecd-b17a-14c09dc1961b",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_dir = \"/home/jiw203/wjn/InstructGraph/output/instruction_tuning/fsdp_peft_flash_1500k/llama2-peft-2epoch/bbh_predictions/all_prediction.json\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "b9549f10-f507-4f5e-9337-bcd0e56fb308",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_dir = \"/home/jiw203/wjn/InstructGraph/output/instruction_tuning/fsdp_peft_flash_1500k/vicuna-peft-2epoch/bbh_predictions/all_prediction.json\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "20fdc049-cdd9-44ad-bb90-ccbbb129adb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_dir = \"/home/jiw203/wjn/InstructGraph/output/instruction_tuning/llama2/bbh_predictions/all_prediction.json\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "82c369d0-1216-4142-a492-23b884d32636",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_dir = \"/home/jiw203/wjn/InstructGraph/output/instruction_tuning/vicuna/bbh_predictions/all_prediction.json\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "f8ada839-6565-4a79-81bd-a8fbfea78d48",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(prediction_dir, \"r\", encoding=\"utf-8\") as fr:\n",
    "    predictions = json.load(fr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "44ba3793-1516-4d01-9357-6a6611803616",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6511"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "02caaa26-0ee1-40c2-aebe-50cf05b23b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(predictions):\n",
    "    acc = 0\n",
    "    for ei, example in enumerate(predictions):\n",
    "        prediction = example[\"prediction\"]\n",
    "        answer = example[\"answer\"][0]\n",
    "        prediction = \"\".join(prediction.split(\"\\n\\nQ\")[4])\n",
    "        # print(prediction)\n",
    "        if \"{}\".format(answer) in prediction[-100:]:\n",
    "            acc += 1.0\n",
    "        # else:\n",
    "        #     print(\"【{}】 - 【{}】 - 【{}】\".format(ei, answer, prediction.split(\"\\n\")[-1]))\n",
    "    print(\"acc={}\".format(round(acc / len(predictions), 4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "bfa7bdf1-8e3e-43db-be5b-8b6450a740de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc=0.2961\n"
     ]
    }
   ],
   "source": [
    "evaluate(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "03a58331-6fc0-4e12-899f-613c3d7b7703",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Solve multi-step arithmetic problems.\n",
      "\n",
      "Q: ((-5 + 9 * -4 - 0) * (4 + -7 + 0 * -5)) =\n",
      "A: Let's think step by step.\n",
      "Let’s recall that the order of operations in mathematics is as follows: (1) Parentheses, (2) exponents, (3) multiplication and division (from left to right), (4) addition and multiplication (from left to right). So, remember to always compute the expressions inside parentheses or brackets first.\n",
      "This equation can be written as \"A * B\", where A = (-5 + 9 * -4 - 0) and B = (4 + -7 + 0 * -5).\n",
      "Let's calculate A = (-5 + 9 * -4 - 0) = (-5 + (9 * -4) - 0) = (-5 + (-36) - 0) = (-5 + -36 - 0) = -5 - 36 = -41.\n",
      "Let's calculate B = (4 + -7 + 0 * -5) = (4 + -7 + (0 * -5)) = (4 + -7 + 0) = (4 + -7) = (4 - 7) = -3.\n",
      "Then, the final equation is A * B = -41 * -3 = (-61) * (-3) = 123. So the answer is 123.\n",
      "\n",
      "Q: ((-9 * 7 * 7 * -9) + (4 * -9 - 8 - -4)) =\n",
      "A: Let's think step by step.\n",
      "Let’s recall that the order of operations in mathematics is as follows: (1) Parentheses, (2) exponents, (3) multiplication and division (from left to right), (4) addition and multiplication (from left to right). So, remember to always compute the expressions inside parentheses or brackets first.\n",
      "This equation can be written as \"A + B\", where A = (-9 * 7 * 7 * -9) and B = (4 * -9 - 8 - -4).\n",
      "Let's calculate A = (-9 * 7 * 7 * -9) = ((-9 * 7) * (7 * -9))  = ((-63) * (-63)) = 3969.\n",
      "Let's calculate B = (4 * -9 - 8 - (-4)) = ((4 * -9) - 8 - (-4)) = ((-36) - 8 - (-4)) = ((-36 - 8) - (-4)) = (-44 - (-4)) = -40.\n",
      "Then, the final equation is A + B = 3969 + -40 = 3969 - 40 = 3929. So the answer is 3929.\n",
      "\n",
      "Q: ((-3 + 5 * 8 * -4) - (9 - 8 * -7 + -9)) =\n",
      "A: Let's think step by step.\n",
      "Let’s recall that the order of operations in mathematics is as follows: (1) Parentheses, (2) exponents, (3) multiplication and division (from left to right), (4) addition and multiplication (from left to right). So, remember to always compute the expressions inside parentheses or brackets first.\n",
      "This equation can be written as \"A - B\", where A = (-3 + 5 * 8 * -4) and B = (9 - 8 * -7 + -9).\n",
      "Let's calculate A = (-3 + 5 * 8 * -4) = (-3 + (5 * 8) * -4) = (-3 + (40) * -4) = (-3 + (40 * -4)) = (-3 + -160) = -163.\n",
      "Let's calculate B = (9 - 8 * -7 + -9) = (9 - (8 * -7) + -9) = (9 - (-56) + -9) = ((9 - (-56)) + -9) = ((65) + -9)= (65 - 9) = 56.\n",
      "Then, the final equation is A - B = -163 - 56 = -219. So the answer is -219.\n",
      "\n",
      "Q: ((6 - 0 * 5 + -3) * (6 - -7 + -2 - -7)) =\n",
      "A: Let's think step by step.\n",
      "Let’s recall that the order of operations in mathematics is as follows: (1) Parentheses, (2) exponents, (3) multiplication and division (from left to right), (4) addition and multiplication (from left to right). So, remember to always compute the expressions inside parentheses or brackets first.\n",
      "This equation can be written as \"A * B\", where A = (6 - 0 * 5 + -3) and B = (6 - -7 + -2 - -7).\n",
      "Let's calculate A = (6 - 0 * 5 + -3) = (6 - (0 * 5) + -3) = (6 - (30) + -3) = (6 - (30 - 3)) = (6 - (27)) = (6 - 27) = -21.\n",
      "Let's calculate B\n"
     ]
    }
   ],
   "source": [
    "print(predictions[4334][\"prediction\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1575402c-ef7d-4eda-94d4-f936f0302197",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "': The following paragraphs each describe a set of five objects arranged in a fixed order. The statements are logically consistent within each paragraph. In a golf tournament, there were five golfers: Dan, Mel, Ana, Eli, and Rob. Rob finished above Mel. Mel finished second. Ana finished above Eli. Dan finished last.\\nOptions:\\n(A) Dan finished second\\n(B) Mel finished second\\n(C) Ana finished second\\n(D) Eli finished second\\n(E) Rob finished second\\nA: Let\\'s think step by step.\\n(1) Rob finished above Mel: \"(above) ? Mel ? Rob ? (below)\".\\n(2) Mel finished second: \"(above) ? Rob ? Mel ? (below)\".\\n(3) Ana finished above Eli: \"(above) ? Eli ? Ana ? (below)\".\\n(4) Dan finished last: \"(above) ? Ana ? Eli ? Dan ? (below)\".\\n(5) Combining (1), (2), (3), and (4) we get the following ordering: \"(above) Ana Mel Dan Rob Eli (below)\".\\nAccording to this ordering, the person who finished second is Mel.\\nMel finished second. So the answer is (B).'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\".join(predictions[1232][\"prediction\"].split(\"\\n\\nQ\")[4])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
